# -*- coding:utf-8 -*-
import codecs
from operator import add

from common_module.pub_utils import is_other


__author__ = 'root'
from django.core.management.base import BaseCommand
import os
from kao_question.models import *
from kao_media.models import *
from common_module.path_handle import source_path_handle, source_file_copy,kaoala_jieba_dict
import logging
log = logging.getLogger('django_crontab.crontab')

from common_module.source_handles.kaoala_source_handle import dataframe_init,proportion_material_table,material_solidify_table
from common_module.source_handles.kaoala_source_datafeici import source_seach_char

def fenci_question():
    question_qbody_dict={}
    question_qhins_dict={}
    question_pic_dcit={}

    answer_answerText_dcit={}
    answer_corrent_dict={}

    video_introduction_dict={}   #考点视频引言
    video_conclusion_dict={}    #考点视频总结

    subject_title_dict={}    #考点名字

    video_dict={}

    word_question_qbody_dict={}
    word_question_qhins_dict={}
    word_answer_answerText_dcit={}
    word_answer_corrent_dict={}
    word_video_introduction_dict={}
    word_video_conclusion_dict={}
    word_subject_dict={}


    resources_tags_dict={}
    resources_type_dict={}   #额外资源的类型
    resources_desc_dict = {}  # 资源简介

    word_resources_tag_dict={}
    resources_subjectID_dict={}    #资源的考点id  key is int   values is list
    resources_title_dict = {}
    word_resources_title_dict={}
    word_resources_desc_dict={}
    word_resources_subject_dict={}

    exam_year_dict={}
    exam_area_dict={}
    word_exam_tag_dict={}



    log.info(u'start 提取题目数据')
    question_list=kao_question.objects.all()
    for sub_question in question_list:
        qbody=sub_question.qbody
        if qbody!=None and qbody!='':
            question_qbody_dict[int(sub_question.id)]=qbody
        qhins=sub_question.qhins
        if qhins!=None and qhins!='':
            question_qhins_dict[int(sub_question.id)]=qhins

        if sub_question.pic!=None and sub_question.pic!='':
            question_pic_dcit[int(sub_question.id)]=sub_question.pic
    log.info(u'end 提取题目数据')

    log.info(u'start 提取答案数据')
    answer_list=kao_answer.objects.all()
    for sub_answer in answer_list:
        if sub_answer.answerText!=None and sub_answer.answerText!='':
            answer_answerText_dcit[int(sub_answer.id)]=sub_answer.answerText
        if sub_answer.corrent_answer!=None and sub_answer.corrent_answer!='':
            answer_corrent_dict[int(sub_answer.id)]=sub_answer.corrent_answer
    log.info(u'end 提取答案数据')

    log.info(u'start 提取视频描述数据')
    desc_list=subject_desc.objects.all()
    for sub_desc in desc_list:
        if sub_desc.video_introduction!=None and sub_desc.video_introduction!='':
            video_introduction_dict[int(sub_desc.subject_id)]=sub_desc.video_introduction

        if sub_desc.video_conclusion!=None and sub_desc.video_conclusion!='':
            video_conclusion_dict[int(sub_desc.subject_id)]=sub_desc.video_conclusion
    log.info(u'end 提取视频描述数据')

    log.info(u'start 提取考点title数据')
    subject_list=test_subjects.objects.filter(flag=2)
    for sub_subject in subject_list:
        if sub_subject.subjects_name!=None and sub_subject.subjects_name!='':
            subject_title_dict[int(sub_subject.id)]=sub_subject.subjects_name
    log.info(u'end 提取考点title数据')

    log.info(u'start 提取视频地址数据')
    video_list=material_manager.objects.filter(mtype=1,purpose=0)
    for sub_video in video_list:
        video_dict[int(sub_video.subject.all()[0].id)]=sub_video.material_url
    log.info(u'end 提取视频地址数据')


    log.info(u'start 提取额外资源数据')
    res_list=resources.objects.all()
    for sub_res in res_list:
        resources_type_dict[int(sub_res.id)]=8
        resources_tags_dict[int(sub_res.id)]=u''.join(map(lambda  x:x['tag_name'],sub_res.resources_tags_set.values('tag_name')))
        resources_title_dict[int(sub_res.id)]=sub_res.resources_name
        resources_desc_dict[int(sub_res.id)]=sub_res.resources_desc
    log.info(u'end 提取额外资源数据')

    log.info(u'start 提取额外资源与考点对应关系数据')
    resource_subject_list=resources_subjects_relationship.objects.all()
    for sub_res_sub in resource_subject_list:
        temp_subject_list=resources_subjectID_dict.get(sub_res_sub.resources_id,[])
        if sub_res_sub.subject_id not in temp_subject_list:
            temp_subject_list.append(sub_res_sub.subject_id)
        resources_subjectID_dict[sub_res_sub.resources_id]=temp_subject_list
    log.info(u'end 提取额外资源与考点对应关系数据')

    log.info(u'start 提取试卷年份数据')
    exam_list = kao_exam.objects.all()
    for sub_time in exam_list:
        exam_year_dict[int(sub_time.id)]=sub_time.year_of_exam.strftime('%Y')
    log.info(u'end 提取试卷年份数据')

    log.info(u'start 提取试卷地区数据')
    for sub_area in exam_list:
        exam_area_dict[int(sub_area.id)]=u''.join(list(set(map(lambda x: x['province'],sub_area.kao_exam_area_set.values('province')))))
    log.info(u'end 提取试卷地区数据')


    log.info(u'start 进行分词')
    all_text_list=question_qbody_dict.values()+question_qhins_dict.values()+answer_answerText_dcit.values()+answer_corrent_dict.values()+video_introduction_dict.values()+video_conclusion_dict.values()+subject_title_dict.values()
    all_text=u''.join(all_text_list)
    items=source_seach_char(seach_text=all_text)
    all_word_list=[]
    if items['code']==0:
        all_word_list=list(set(items['data']))
    log.info(u'end 进行分词')

    log.info(u'start 额外资源分词')
    all_res_text=u''.join(resources_tags_dict.values())
    all_res_text+=u''.join(resources_title_dict.values())
    all_res_text+=u''.join(resources_desc_dict.values())
    items=source_seach_char(seach_text=all_res_text)
    if items['code']==0:
        all_word_list+=list(set(items['data']))
    log.info(u'end 额外资源分词')


    log.info(u'start 对题目qbody进行分词')
    for key,value in question_qbody_dict.items():
        new_str=u''
        for sub_key in value:
            if is_other(sub_key)==True:  #判断单个字符是否为符号
                continue
            else:
                new_str=new_str+sub_key
        one_word_list=[]
        for sub_word in all_word_list:
            if sub_word in new_str:
                one_word_list.append(sub_word)
        word_question_qbody_dict[key]=one_word_list
    log.info(u'end 对题目qbody进行分词')

    log.info(u'start 对题目qhins进行分词')
    for key,value in question_qhins_dict.items():
        new_str=u''
        for sub_key in value:
            if is_other(sub_key)==True:  #判断单个字符是否为符号
                continue
            else:
                new_str=new_str+sub_key
        one_word_list=[]
        for sub_word in all_word_list:
            if sub_word in new_str:
                one_word_list.append(sub_word)
        word_question_qhins_dict[key]=one_word_list
    log.info(u'end 对题目qhins进行分词')


    log.info(u'start 对答案answerText进行分词')
    for key,value in answer_answerText_dcit.items():
        new_str=u''
        for sub_key in value:
            if is_other(sub_key)==True:  #判断单个字符是否为符号
                continue
            else:
                new_str=new_str+sub_key
        one_word_list=[]
        for sub_word in all_word_list:
            if sub_word in new_str:
                one_word_list.append(sub_word)
        word_answer_answerText_dcit[key]=one_word_list
    log.info(u'end 对答案answerText进行分词')


    log.info(u'start 对答案corrent进行分词')
    for key,value in answer_corrent_dict.items():
        new_str=u''
        for sub_key in value:
            if is_other(sub_key)==True:  #判断单个字符是否为符号
                continue
            else:
                new_str=new_str+sub_key
        one_word_list=[]
        for sub_word in all_word_list:
            if sub_word in new_str:
                one_word_list.append(sub_word)
        word_answer_corrent_dict[key]=one_word_list
    log.info(u'end 对答案corrent进行分词')

    log.info(u'start 对考点title进行分词')
    for key, value in subject_title_dict.items():
        new_str = u''
        for sub_key in value:
            if is_other(sub_key) == True:  # 判断单个字符是否为符号
                continue
            else:
                new_str = new_str + sub_key
        one_word_list = []
        for sub_word in all_word_list:
            if sub_word in new_str:
                one_word_list.append(sub_word)
            word_subject_dict[key] = one_word_list
    log.info(u'end 对考点title进行分词')


    log.info(u'start 对视频引入进行分词')
    for key,value in video_introduction_dict.items():
        new_str=u''
        for sub_key in value:
            if is_other(sub_key)==True:  #判断单个字符是否为符号
                continue
            else:
                new_str=new_str+sub_key
        one_word_list=[]
        for sub_word in all_word_list:
            if sub_word in new_str:
                one_word_list.append(sub_word)
            word_video_introduction_dict[key]=one_word_list
    log.info(u'end 对视频引入进行分词')

    log.info(u'start 对视频学以致用进行分词')
    for key, value in video_conclusion_dict.items():
        new_str = u''
        for sub_key in value:
            if is_other(sub_key) == True:  # 判断单个字符是否为符号
                continue
            else:
                new_str = new_str + sub_key
        one_word_list = []
        for sub_word in all_word_list:
            if sub_word in new_str:
                one_word_list.append(sub_word)
            word_video_conclusion_dict[key] = one_word_list
    log.info(u'end 对视频学以致用进行分词')

    log.info(u'start 对额外资源tags标签进行分词')
    for key,value in resources_tags_dict.items():
        new_str=u''
        for sub_key in value:
            if is_other(sub_key)==True:  #判断单个字符是否为符号
                continue
            else:
                new_str=new_str+sub_key
        one_word_list=[]
        for sub_word in all_word_list:
            if sub_word in new_str:
                one_word_list.append(sub_word)
        word_resources_tag_dict[key]=one_word_list
    log.info(u'start 对额外资源tags标签进行分词')

    log.info(u'start 对额外资源title进行分词')
    for key, value in resources_title_dict.items():
        new_str = u''
        for sub_key in value:
            if is_other(sub_key) == True:  # 判断单个字符是否为符号
                continue
            else:
                new_str = new_str + sub_key
        one_word_list = []
        for sub_word in all_word_list:
            if sub_word in new_str:
                one_word_list.append(sub_word)
        word_resources_title_dict[key] = one_word_list
    log.info(u'start 对额外资源title进行分词')

    log.info(u'start 对额外资源简介进行分词')
    for key, value in resources_desc_dict.items():
        new_str = u''
        for sub_key in value:
            if is_other(sub_key) == True:  # 判断单个字符是否为符号
                continue
            else:
                new_str = new_str + sub_key
        one_word_list = []
        for sub_word in all_word_list:
            if sub_word in new_str:
                one_word_list.append(sub_word)
            word_resources_desc_dict[key] = one_word_list
    log.info(u'start 对额外资源简介进行分词')

    log.info(u'start 对额外资源考点进行分词')
    for key,value in resources_subjectID_dict.items():
        temp_word_list=map(lambda x:word_subject_dict.get(x),value)
        word_resources_subject_dict[key]=list(set(reduce(add,temp_word_list)))
    log.info(u'end 对额外资源考点进行分词')

    log.info(u'start 试卷年份和地区的分词')
    all_word_list += exam_year_dict.values()
    all_area_text = u''.join(exam_area_dict.values())
    items = source_seach_char(seach_text=all_area_text)
    if items['code'] == 0:
        all_word_list += list(set(items['data']))

    for key,value in exam_year_dict.items():
        word_exam_tag_dict[key]=[value]

    for key,value in exam_area_dict.items():
        new_str = u''
        for sub_key in value:
            if is_other(sub_key) == True:  # 判断单个字符是否为符号
                continue
            else:
                new_str = new_str + sub_key
        one_word_list = []
        for sub_word in all_word_list:
            if sub_word in new_str:
                one_word_list.append(sub_word)
        temp_tags=word_exam_tag_dict.get(key,[])
        temp_tags+=one_word_list
        word_exam_tag_dict[key]=temp_tags
    log.info(u'end 试卷年份和地区的分词')



    log.info(u'start 加载所有的表的dataframe')
    dataframe_init_obj=dataframe_init(word_dict_path=kaoala_jieba_dict)
    dataframe_init_obj.load_ram()
    log.info(u'end 加载所有的表的dataframe')


    log.info(u'start 对权重与素材映射进行处理')
    proportion_material_table_obj=proportion_material_table(dataframe_init_obj.word_dataframe,dataframe_init_obj.word_material_proportion_dataframe,dataframe_init_obj.proportion_material_6_dataframe)
    material_solidify_table_obj=material_solidify_table(dataframe_init_obj.material_solidify_dataframe)
    #log.info(u'self.proportion_material_6_dataframe  内存大小 %s MB' %int(proportion_material_table_obj.proportion_material_6_dataframe.memory_usage(index=True).sum()/1024/1024))
    dataframe_init_obj.release_ram()

    log.info(u'start word_question_qbody_dict 素材映射处理')
    for key,value in word_question_qbody_dict.items():
        proportion_material_table_obj.write_proportion_material(word_list=value,p_category=1,material_id=key,show_level=0)
        #log.info(u'self.proportion_material_6_dataframe  内存大小 %s MB' %int(proportion_material_table_obj.proportion_material_6_dataframe.memory_usage(index=True).sum()/1024/1024))
    log.info(u'end word_question_qbody_dict 素材映射处理')

    log.info(u'start word_question_qhins_dict 素材映射处理')
    for key,value in word_question_qhins_dict.items():
        proportion_material_table_obj.write_proportion_material(word_list=value,p_category=3,material_id=key,show_level=0)
    log.info(u'end word_question_qhins_dict 素材映射处理')

    log.info(u'start question_pic_dcit 素材映射处理')
    for key,value in question_pic_dcit.items():
        pic_word_list=word_question_qbody_dict.get(key,[])+word_question_qhins_dict.get(key,[])
        pic_word_list=list(set(pic_word_list))
        proportion_material_table_obj.write_proportion_material(word_list=pic_word_list,p_category=2,material_id=key,show_level=0)
    log.info(u'end question_pic_dcit 素材映射处理')

    log.info(u'start word_answer_answerText_dcit 素材映射处理')
    for key,value in word_answer_answerText_dcit.items():
        proportion_material_table_obj.write_proportion_material(word_list=value,p_category=4,material_id=key,show_level=0)
    log.info(u'end word_answer_answerText_dcit 素材映射处理')

    log.info(u'start word_answer_corrent_dict 素材映射处理')
    for key,value in word_answer_corrent_dict.items():
        proportion_material_table_obj.write_proportion_material(word_list=value,p_category=4,material_id=key,show_level=0)
    log.info(u'end word_answer_corrent_dict 素材映射处理')

    log.info(u'start word_subject_dict 素材映射处理')
    for key,value in word_subject_dict.items():
        proportion_material_table_obj.write_proportion_material(word_list=value, p_category=6, material_id=key,show_level=4)
    log.info(u'end word_subject_dict 素材映射处理')

    log.info(u'start word_video_introduction_dict 素材映射处理')
    for key,value in word_video_introduction_dict.items():
        proportion_material_table_obj.write_proportion_material(word_list=value,p_category=6,material_id=key,show_level=5)
    log.info(u'end word_video_introduction_dict 素材映射处理')

    log.info(u'start word_video_conclusion_dict 素材映射处理')
    for key, value in word_video_conclusion_dict.items():
        proportion_material_table_obj.write_proportion_material(word_list=value, p_category=6, material_id=key,show_level=6)
    log.info(u'end word_video_conclusion_dict 素材映射处理')

    log.info(u'start word_resources_tag_dict 素材映射处理')
    for key,value in word_resources_tag_dict.items():
        p_category=resources_type_dict.get(key,8)
        proportion_material_table_obj.write_proportion_material(word_list=value,p_category=p_category,material_id=key,show_level=2)
    log.info(u'end word_resources_tag_dict 素材映射处理')

    log.info(u'start word_resources_title_dict 素材映射处理')
    for key, value in word_resources_title_dict.items():
        p_category = resources_type_dict.get(key, 8)
        proportion_material_table_obj.write_proportion_material(word_list=value, p_category=p_category, material_id=key,show_level=1)
    log.info(u'end word_resources_title_dict 素材映射处理')

    log.info(u'start word_resources_desc_dict 素材映射处理')
    for key,value in word_resources_desc_dict.items():
        p_category = resources_type_dict.get(key, 8)
        proportion_material_table_obj.write_proportion_material(word_list=value, p_category=p_category, material_id=key,show_level=3)
    log.info(u'end word_resources_desc_dict 素材映射处理')

    log.info(u'start word_resources_subject_dict 素材映射处理')
    for key,value in word_resources_subject_dict.items():
        p_category = resources_type_dict.get(key, 8)
        proportion_material_table_obj.write_proportion_material(word_list=value, p_category=p_category, material_id=key,show_level=4)
    log.info(u'end word_resources_subject_dict 素材映射处理')

    log.info(u'start word_exam_tag_dict 试卷映射处理')
    for key,value in word_exam_tag_dict.items():
        proportion_material_table_obj.write_proportion_material(word_list=value,p_category=9,material_id=key,show_level=0)
    log.info(u'end word_exam_tag_dict 试卷映射处理')



    proportion_material_table_obj.solidify()
    proportion_material_table_obj.release_ram()
    log.info(u'end 对权重与素材映射进行处理')

    log.info(u'start 对素材表进行处理')
    log.info(u'start question_qbody_dict 素材处理')
    for key,value in question_qbody_dict.items():
        material_solidify_table_obj.write_material_solidify(material_id=key,p_category=1)
    log.info(u'end question_qbody_dict 素材处理')

    log.info(u'start question_qhins_dict 素材处理')
    for key,value in question_qhins_dict.items():
        material_solidify_table_obj.write_material_solidify(material_id=key,p_category=3)
    log.info(u'end question_qhins_dict 素材处理')

    log.info(u'start question_pic_dcit 素材处理')
    for key,value in question_pic_dcit.items():
        material_solidify_table_obj.write_material_solidify(material_id=key,p_category=2,pic_meida_path=value)
    log.info(u'end question_pic_dcit 素材处理')

    log.info(u'start answer_answerText_dcit 素材处理')
    for key,value in answer_answerText_dcit.items():
        material_solidify_table_obj.write_material_solidify(material_id=key,p_category=4)
    log.info(u'end answer_answerText_dcit 素材处理')

    log.info(u'start answer_corrent_dict 素材处理')
    for key,value in answer_corrent_dict.items():
        material_solidify_table_obj.write_material_solidify(material_id=key,p_category=4)
    log.info(u'end answer_corrent_dict 素材处理')

    log.info(u'start video_desc_dict 素材处理')
    for key,value in subject_title_dict.items():
        material_solidify_table_obj.write_material_solidify(material_id=key, p_category=6,pic_meida_path=video_dict.get(key, ''))
    log.info(u'end video_desc_dict 素材处理')

    log.info(u'start video_introduction_dict 素材处理')
    for key,value in video_introduction_dict.items():
        material_solidify_table_obj.write_material_solidify(material_id=key,p_category=6,pic_meida_path=video_dict.get(key,''))
    log.info(u'end video_introduction_dict 素材处理')

    log.info(u'start video_conclusion_dict 素材处理')
    for key, value in video_conclusion_dict.items():
        material_solidify_table_obj.write_material_solidify(material_id=key, p_category=6,pic_meida_path=video_dict.get(key, ''))
    log.info(u'end video_conclusion_dict 素材处理')

    log.info(u'start resources_tags_dict 素材处理')
    for key,value in resources_tags_dict.items():
        p_category=resources_type_dict.get(key,8)
        material_solidify_table_obj.write_material_solidify(material_id=key,p_category=p_category)
    log.info(u'end resources_tags_dict 素材处理')

    log.info(u'start resources_title_dict 素材处理')
    for key, value in resources_title_dict.items():
        p_category = resources_type_dict.get(key, 8)
        material_solidify_table_obj.write_material_solidify(material_id=key, p_category=p_category)
    log.info(u'end resources_title_dict 素材处理')

    log.info(u'start resources_desc_dict 素材处理')
    for key, value in resources_desc_dict.items():
        p_category = resources_type_dict.get(key, 8)
        material_solidify_table_obj.write_material_solidify(material_id=key, p_category=p_category)
    log.info(u'end resources_desc_dict 素材处理')

    log.info(u'start resources_subjectID_dict 素材处理')
    for key, value in resources_subjectID_dict.items():
        p_category = resources_type_dict.get(key, 8)
        material_solidify_table_obj.write_material_solidify(material_id=key, p_category=p_category)
    log.info(u'end resources_subjectID_dict 素材处理')

    log.info(u'start word_exam_tag_dict 素材处理')
    for key, value in word_exam_tag_dict.items():
        material_solidify_table_obj.write_material_solidify(material_id=key,p_category=9)
    log.info(u'end word_exam_tag_dict 素材处理')



    material_solidify_table_obj.solidify()
    log.info(u'end 对素材表进行处理')
























class Command(BaseCommand):
    def handle(self, *args, **options):
        log.info(u'start 数据更新')

        log.info(u'start 进行文件夹处理')
        source_path_handle()
        log.info(u'end 文件夹处理')

        fenci_question()

        # log.info(u'start 分词词汇固化')
        # word_dict_obj=word_dict(word_dict_path=kaoala_jieba_dict)
        # word_dict_obj.writ_word_dict()
        # log.info(u'end 分词词汇固化')

        # log.info(u'start 固化分词与分词之前比重表')
        # word_word_proportion_obj=word_word_proportion()
        # word_word_proportion_obj.wird_proportion()
        # log.info(u'end 固化分词与分词之前比重表')

        # log.info(u'start 处理素材与分词进行对应')
        # fenci_question()
        # log.info(u'end 处理素材与分词进行对应')

        log.info(u'start 把处理好的源数据文件复制到前台文件夹中')
        source_file_copy()
        log.info(u'end 复制到前台文件夹中')

        log.info(u'end 数据更新')
